package Crawler;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.Properties;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import Client.DBTransaction;
import Crawler.Structure.RegexExpressions;
import Crawler.Structure.RegexExpressions.Expression;
import Structure.MeaningWordPair;
import Structure.Translation;
import Structure.Word;

public class GermanCrawler {

	// word content

	private final HashMap<String, String> langCodes;
	String file;
	public LinkedList<Word> words = new LinkedList<Word>();

	private String languageOfContent;

	// statistic values

	public int skipped = 0;
	private final int maxNumberOfWords = 0;

	// parsing content

	private final RegexExpressions expr = new RegexExpressions();

	private final String pPageB = "\\s*<page>\\s*";
	private final String pPageE = "\\s*.*?</page>\\s*";
	private final Pattern pWord = Pattern
			.compile("\\s*<title>([^:]*)</title>\\s*");

	/**
	 * Start regex for the not xml text 1 rest
	 */
	private final Pattern pTextB = Pattern
			.compile("\\s*<text xml:space=\"preserve\">(.*)");
	/**
	 * end regex for the not xml text
	 */
	private final String pTextE = ".*</text>";

	/**
	 * pattern to match the origin language in the not xml text
	 */
	private Pattern pLanguage;
	// =Pattern.compile("\\s*== (.*) \\(\\{\\{Sprache\\|(\\w*)\\}\\}\\) ==\\s*");

	/**
	 * pattern to match the subcategory after that the translation lines begin
	 * ich weiss "Ue" w�re besser ab so stehts im text!
	 */
	private Pattern pUebersetzung;
	// = "\\s*==== �bersetzungen ====\\s*";

	/**
	 * 1 language 2 rest "{{en}}:......."
	 */
	private Pattern pBeginTranslation; // = Pattern
	// .compile("\\s*\\*\\{\\{(\\w*)\\}\\}:\\s*(.*)");

	private String pEndTranslationChapter;// = "\\s*\\{\\{Referenzen\\}\\}\\s*";

	// functions

	public GermanCrawler(String file, String fileProperties) {

		this.file = file;
		this.expr.loadFromProperties(fileProperties);
		this.langCodes = new HashMap<String, String>();

		// read langCodes from File and write them into the Map
		try {
			BufferedReader bf = new BufferedReader(new FileReader(
					"properties/deLangCodes.txt"));
			String tmp;
			while ((tmp = bf.readLine()) != null) {
				String[] splits = tmp.split(";");
				langCodes.put(splits[0], splits[1]);
			}

		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();

		}

		// load the other regex from properties file
		Properties p = new Properties();
		try {
			BufferedReader bf = new BufferedReader(new FileReader(
					fileProperties));
			p.load(bf);

			pUebersetzung = Pattern.compile(p.getProperty("Uebersetzung"));
			languageOfContent = p.getProperty("languageOfContent");
			pLanguage = Pattern.compile(p.getProperty("pLanguage"));

			pBeginTranslation = Pattern.compile(p
					.getProperty("pBeginTranslation"));

			pEndTranslationChapter = p.getProperty("pEndTranslationChapter");

		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();

		}

	}

	public void parse() {
		int i = 0;
		try {
			BufferedReader bf = new BufferedReader(new FileReader(
					new File(file)));

			String line;
			while ((line = bf.readLine()) != null) {
				if (line.matches(pPageB)) {

					Word w = new Word("", "", languageOfContent);
					parsePage(pPageE, bf, w);
					if (!w.word.matches("") && !w.language.matches("")) {
						i++;
						words.add(w);
						// System.out.println(w);
						if (i == maxNumberOfWords)
							break;
						if (i % 10000 == 0)
							System.out.println(i + " words found");
					} else
						skipped++;
				}
			}

		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	private void parsePage(String endPage, BufferedReader bf, Word w)
			throws IOException {
		String line;

		while ((line = bf.readLine()) != null) {
			if (line.matches(endPage))
				return;

			Matcher m = pWord.matcher(line);
			if (m.matches())
				w.word = m.group(1);

			m = pTextB.matcher(line);
			if (m.matches()) {
				String rest = m.group(1);
				parseSubText(pTextE, bf, w, rest);
			}

		}
	}

	/**
	 * this function parses the subtext of a page. The subtext is that text,
	 * that is not in xml format
	 * 
	 * @param endSubtext
	 *            the pattern that maches, when the subtext ends
	 * @param bf
	 *            to read lines
	 * @param w
	 *            the word where it fills the information
	 * @param rest
	 *            the rest - if there is a rest
	 * @throws IOException
	 */
	public void parseSubText(String endSubtext, BufferedReader bf, Word w,
			String rest) throws IOException {

		String line = rest;

		// contains the translation for the different languages
		HashMap<String, Translation> language = new HashMap<String, Translation>();

		// loop for parsing subtext
		do {
			if (line.matches(endSubtext))
				break;

			// parse the origin language
			Matcher mSp = pLanguage.matcher(line);
			if (mSp.matches()) {
				w.language = this.langCodes.get(mSp.group(2));
				if (w.language == null) {
					// System.out.println("!" + mSp.group(2));
					w.language = mSp.group(2);
				}
			}

			// read translations from the translation chapter
			if (pUebersetzung.matcher(line).matches()) {

				do {
					// if end of chapter translation in subtext
					if (line.matches(pEndTranslationChapter))
						break;
					// if no chapter after Translation
					if (line.matches(endSubtext))
						break;

					// translation line
					Matcher m = pBeginTranslation.matcher(line);
					if (m.matches()) {

						// add new translation language if not exists
						if (!language.containsKey(m.group(1))) {
							language.put(m.group(1), new Translation(
									m.group(1),
									new LinkedList<MeaningWordPair>()));
						}
						// open translation
						Translation t = language.get(m.group(1));
						// System.out.println(t);

						// initialize information for analysis
						String part = m.group(2);

						// parses the rest of the line and add the information
						// to the translation t
						parseTranslationLine(t, part);
					}

				} while ((line = bf.readLine()) != null);

				// if no chapter after Translation
				if (line.matches(endSubtext))
					break;
			}

		} while ((line = bf.readLine()) != null);

		// add translations
		Set<String> keys = language.keySet();
		w.translations = new Translation[keys.size()];
		Iterator<String> it = keys.iterator();
		int i = 0;
		while (it.hasNext()) {
			w.translations[i] = language.get(it.next());
			i++;
		}

	}

	/**
	 * parses a line with the information about the translation of a word.
	 * 
	 * @param t
	 * @param meaning
	 * @param line
	 */
	public void parseTranslationLine(Translation t, String line) {
		String meaning = "";
		String part = line;

		boolean match = true;
		int type = -1;
		while (match) {
			match = false;

			// first run
			if (type == -1) {
				for (int i = 0; i < expr.expressions.size() && match == false; i++) {
					Iterator<Expression> it = expr.expressions.get(i).nextPattern
							.iterator();
					// test all types of translation formats/matcher
					while (it.hasNext() && match == false) {
						Expression e = it.next();
						Matcher mTN = e.pattern.matcher(part);

						if (mTN.matches()) {
							meaning = mTN.group(e.bedeutung);
							t.meaningWordMap.add(new MeaningWordPair(Integer
									.parseInt(meaning), mTN.group(e.word)));
							part = mTN.group(e.rest);
							match = true;
							type = i;
						}
					}
				}

			} else {
				// next part
				Iterator<Expression> it = expr.expressions.get(type).nextPattern
						.iterator();
				while (it.hasNext() && match == false) {
					Expression e = it.next();
					Matcher mTN = e.pattern.matcher(part);
					if (mTN.matches()) {
						meaning = mTN.group(e.bedeutung);
						t.meaningWordMap.add(new MeaningWordPair(Integer
								.parseInt(meaning), mTN.group(e.word)));
						part = mTN.group(e.rest);
						match = true;

					}
				}

				// same part
				if (match == false) {
					it = expr.expressions.get(type).samePattern.iterator();
					while (it.hasNext() && match == false) {
						Expression e = it.next();
						Matcher mT1S = e.pattern.matcher(part);
						if (mT1S.matches() && meaning != "") {
							t.meaningWordMap.add(new MeaningWordPair(Integer
									.parseInt(meaning), mT1S.group(e.word)));
							part = mT1S.group(e.rest);
							match = true;
						}
					}
				}
			}
		}
	}

	public void addWordsToDB(LinkedList<Word> words) {
		DBTransaction t = new DBTransaction();
		t.connect();
		t.addWord(words);
		// for (int i = 0; i < words.size(); i++){
		// t.addWord(words.get(i));
		// if(i%10000==0)
		// System.out.println("currently added: "+ i);
		// }
	}

	/**
	 * @param args
	 */
	public static void main(String[] args) {

		GermanCrawler gc = new GermanCrawler(
				// "example.xml","DECrawler.properties");
				// GermanCrawler gc = new GermanCrawler(
				"dewiktionary-20111124-pages-articles.xml",
				"DECrawler.properties");

		gc.parse();

		// for (int i = 0; i < gc.words.size(); i++)
		// System.out.println(gc.words.get(i));

		System.out.println("\nskipped:\t" + gc.skipped);
		System.out.println("words:\t" + gc.words.size());

		// add to db
		gc.addWordsToDB(gc.words);

		// Translation t= new Translation("en", new LinkedList<Integer>(), new
		// LinkedList<String>());
		//
		// gc.parseTranslationLine(t,
		// " [1] {{�|en|chair}}; [2] {{�|en|stool}}, {{�|en|feces}}; [3] {{�|en|bureau}} ");
		// System.out.println(t);
	}

}
